setup

This is the introduction paragraph. In this file we’re cleaning the data and setting up files for later processing.

Load, clean, recode and save

Load files

# Load the neighborhoods GeoJSON
# creates data["Parcels"] containing geojson data.
data = {}
for feature in [selector] + features:
    geofile = get_newest_file( precious_folder, feature, ext=".geojson" )
    #logger.debug(geofile)
    logger.info(f"Found {feature}:  {geofile}" )
    data[feature] = gpd.read_file( geofile )

# convert all feature files to same CRS mapping as Civic_Associations
for feature in features:
    data[feature] = data[feature].to_crs( data[selector].crs )

INFO     | __main__:<module>:7 - Found Civic_Associations:  /home/john/projects/ssg-thefan-analysis/precious/Civic_Associations-2025-05-16.geojson
INFO     | __main__:<module>:7 - Found Addresses:  /home/john/projects/ssg-thefan-analysis/precious/Addresses-2025-09-25.geojson
INFO     | __main__:<module>:7 - Found Parcels:  /home/john/projects/ssg-thefan-analysis/precious/Parcels-2025-09-25.geojson

Drop columns

# columns to drop:
shared_drops = ['CreatedBy','CreatedDate','EditBy','EditDate']
## file specific column drop mappings
drop_columns = {
    "Civic_Associations" : ['OBJECTID'] + shared_drops,
    "Addresses"          : ['OBJECTID'] + shared_drops,
    "Parcels"            : ['OBJECTID'],
}
for feature in [selector] + features:
    data[feature] = data[feature].drop(columns=drop_columns[feature])

Spatial join, select only parcels and addresses in the Fan.

# Pull out only FDA from Civic_Associations and store it
data[selector_key] = data[selector][ data[selector]["Name"] == selector_key ]

# Select only features from the selector_key (FDA).  perform spatial join.
for feature in features:
    predicate = "overlaps" if feature=="Neighborhoods" else "within"
    data[feature+"_in_fan"] = gpd.sjoin(data[feature], data[selector_key], predicate=predicate, how="inner")

Drop unnecessary columns

# Drop columns created from spatial join
shared_drops = ["index_right","AdoptionDate","ChangeDate","Shape__Area","Shape__Length"]
drop_columns = {
    "Addresses"          : ['GlobalID'] + shared_drops,
    "Parcels"            : ['MaskedOwner','GlobalID_left','GlobalID_right'] + shared_drops,
}
for feature in features:
    feature_name = feature+"_in_fan"
    data[feature_name] = data[feature_name].drop(columns=drop_columns[feature])

Create columns and recode columns in Parcels

gdf = data["Parcels_in_fan"]

gdf["OwnerOccupied"] = gdf.apply(
    lambda row: str(row["MailAddress"]).startswith(str(row["AsrLocationBldgNo"]))
                and str(row["MailCity"]).upper() == "RICHMOND"
                and str(row["MailState"]).upper() == "VA"
                and str(row["MailZip"]) == "23220",
    axis=1
).map({True: 1, False: 0})


# Rule 1: If PropertyClass contains 'Commercial'
gdf.loc[gdf["PropertyClass"].str.contains("Commercial", case=False, na=False), "LandUse"] = "Commercial"

# Rule 2: If PropertyClass contains 'Condo'
gdf.loc[gdf["PropertyClass"].str.contains("Condo", case=False, na=False), "LandUse"] = "Multi-Family"

gdf["SharedGeometry"] = gdf.duplicated(subset="geometry", keep=False).astype(int)

# Create FanUse

mapping = {
    "Single Family": "FanResidential",
    "Multi-Family": "FanResidential",
    "Duplex (2 Family)" : "FanResidential",
    "Commercial": "FanBusiness",
    "Industrial": "FanBusiness",
    "Office" : "FanBusiness",
    "Institutional" : "FanBusiness",
    "Mixed-Use" : "FanMixed-Use"
}
gdf["FanUse"] = gdf["LandUse"].map(mapping).fillna("FanOther")

# Ensure PropertyClass is string and safe for NaNs
mask = gdf["PropertyClass"].fillna("").str.contains("vacant|parking|common|garage|storage|tower|space", case=False, na=False)
# Apply recode
gdf.loc[mask, "FanUse"] = "FanOther"


# A new variable to permit easy selecting

gdf["FanUseType"] = "FanIgnore"  # default
gdf.loc[(gdf["OwnerOccupied"] == 1) & (gdf["Mailable"] == 1), "FanUseType"] = "FanOwner"
gdf.loc[(gdf["OwnerOccupied"] == 0) & (gdf["Mailable"] == 1), "FanUseType"] = "FanRental"

# Reset if it's one of the FanOther property classes.
gdf.loc[mask, "FanUseType"] = "FanIgnore"


mapping = {
    "FanResidential": 1,
    "FanBusiness": 10,
    "FanMixed-Use" : 20,
    "FanOther": 99
}
gdf["FanUseOrder"] = gdf["FanUse"].map(mapping).fillna(99)

data["Parcels_in_fan"] = gdf

Clean columns in Addresses

Save Parcels_in_fan and Addresses_in_fan for later use.

for feature in features:
    feature_name = feature+"_in_fan"

    # store to parquet using pyarror (workflow tip from chatgpt
    data[feature_name].to_parquet(f"{feature_name}.parquet",engine="pyarrow")
    logger.info(f"Saving: {feature_name}.parquet" )

    # create dataframe without spatial geometries and store to CSV
    gdf = data[feature_name].drop(columns="geometry")
    gdf.to_csv(f"{feature_name}.csv", index=False)
    logger.info(f"Saving: {feature_name}.csv" )

INFO     | __main__:<module>:6 - Saving: Addresses_in_fan.parquet
INFO     | __main__:<module>:11 - Saving: Addresses_in_fan.csv
INFO     | __main__:<module>:6 - Saving: Parcels_in_fan.parquet
INFO     | __main__:<module>:11 - Saving: Parcels_in_fan.csv

Examine Addresses

In the following sections we review the Addresses file. If changes are necessary, iterate with the cleaning sections and rerun the report until everyting is clean.